package main

import (
	"flag"
	"log"

	"gitee.com/Flearning/go_crawl/mgobson"
	"gitee.com/Flearning/go_crawl/utils/strip"
	"gitee.com/Flearning/go_crawl/xueqiu/sh01"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	mgoXueqiuCollection string = "crawl_xueqiu_statuses"
	stripTagsCollection string = "strip_tags_data"
	limit               int
	lastAt              int64
)

func init() {
	flag.IntVar(&limit, "limit", 10, "slect limit var.")
}

func main() {
	flag.Parse()

	var (
		maxPage int
	)
	mgobson.NewInit()

	mgobson.MgoSession = mgobson.CopyMgoConn()
	//c := mgobson.MgoSession.DB(mgobson.AuthDatabase).C(stripTagsCollection)
	lastAt = getLastAt()
	log.Printf("This strip data LastAt: %d", lastAt)
	crawl_data_len := getCount()
	// limit := 5
	if crawl_data_len%limit > 0 {
		maxPage = crawl_data_len/limit + 1
	} else {
		maxPage = crawl_data_len / limit
	}
	log.Printf("This data count: %d , MaxPage: %d\n", crawl_data_len, maxPage)

	for i := 0; i <= maxPage; i++ {
		res := getList(i)
		log.Printf("This Page: %d \n", i)
		newRes := make([]interface{}, 0)
		//		for n := 0; n < len(res); n++ {
		//			res[n].Description = stripTagsData(res[n].Description)
		//			res[n].Text = stripTagsData(res[n].Text)
		//			newRes = append(newRes, res[n])
		//		}
		for key, val := range res {
			res[key].Description = stripTagsData(val.Description)
			res[key].Text = stripTagsData(val.Text)
			newRes = append(newRes, res[key])
		}

		log.Println("newRes:", len(newRes))
		//log.Println("lenRes:", len(res))
		// log.Panicln(res...)
		runInstert(newRes...)
	}
	log.Printf("This maxPage: %d Comm exit. \n", maxPage)
}

func stripTagsData(str string) string {
	return strip.StripTags(str)
}

func getList(skip int) []sh01.ListStruct {
	c := mgobson.MgoSession.DB(mgobson.AuthDatabase).C(mgoXueqiuCollection)
	sls := make([]sh01.ListStruct, 0)
	if skip <= 0 {
		skip = 0
	} else {
		skip = skip * limit
	}
	query := bson.M{
		"created_at": bson.M{
			"$gt": lastAt,
		},
	}
	c.Find(query).Sort("-created_at").Skip(skip).Limit(limit).All(&sls)
	return sls
}

func getCount() int {
	c := mgobson.MgoSession.DB(mgobson.AuthDatabase).C(mgoXueqiuCollection)
	query := bson.M{
		"created_at": bson.M{
			"$gt": lastAt,
		},
	}
	q := c.Find(query)
	t, _ := q.Count()
	return t
}

func getLastAt() int64 {
	sls := new(sh01.ListStruct)
	c := mgobson.MgoSession.DB(mgobson.AuthDatabase).C(stripTagsCollection)
	c.Find(nil).Sort("-created_at").One(&sls)
	lastAt = int64(sls.CreatedAt)
	return lastAt
}

func runInstert(v ...interface{}) error {
	query := func(c *mgo.Collection) error {
		return c.Insert(v...)
	}
	return mgobson.WatchCollection(stripTagsCollection, query)
}
